package org.lc.httpcollector

import kotlinx.coroutines.delay
import org.lc.httpcollector.model.Server
import java.util.*
import java.util.concurrent.ConcurrentSkipListSet
import java.util.concurrent.LinkedBlockingQueue

/**
 * http接口采集器
 */
class Collector(private val server: Server) {
    suspend fun collect() {
        onCollectStarting(server)
        while (true) {
            val url = urlQueue.poll()?.removeSuffix("/") ?: break
            if (usedUrlSet.contains(url)) {
                continue
            }
            usedUrlSet.add(url)
            val response = sendRequest(server, url)
            if (responseFilter(server, url, response)) {
                saveFile(server, url, response)
                urlQueue.addAll(parseNextUrls(response.body))
            }
            if (delayMs > 0) delay(delayMs)
        }
        onCollectFinish(server)
    }

    private val urlQueue: Queue<String> = LinkedBlockingQueue<String>().apply { addAll(baseUrls) }
    private val usedUrlSet = ConcurrentSkipListSet<String>()
}